"""
Obtains average user hours spent on namespaces, by cohort and offset

"""

import os
import sys
import multiprocessing
import datetime
import traceback
import math
import collections

def main():

	# The number of worker processes the worker pool will use to process the work queue.
	# Recommend setting this to half the number of available cores to avoid hogging resources.
	# That would be 24 cores on platinum.
	# Once you get above a certain level of CPU usage, I/O throughput will bottleneck anyway.
	NUM_WORKERS = 24
	
	# The number of translation tasks given to each process at a time.
	# Recommend this setting to be 1 for maximum throughput.
	# If I/O throughput ever becomes an issue, increasing this value will make non-CPU bound tasks run more efficiently.
	TASK_PARTITION_SIZE = 1
	
	inputDir = sys.argv[1]
	outputDir = sys.argv[2]

	inputFileList = os.listdir(inputDir)

	filePathArgs = []
	filesRemoved = False

	for ffile in os.listdir(outputDir):
		os.remove(outputDir+ffile)
		filesRemoved =True
	if filesRemoved:
		print "Previous files have been deleted."

	global nsList
	nsList = ["main", "talk", "user", "user talk", "project", "project talk",
		"file", "file talk", "mediawiki", "mediawiki talk", "template", "template talk", "help", "help talk",
		"category", "category talk", "portal", "portal talk", "book", "book talk"]
	
	for inputFileName in inputFileList:
		
		inputFilePath = inputDir+'/'+inputFileName
		outputFilePath = outputDir+'/'+inputFileName
		
		if os.path.isfile(inputFilePath):
			filePathArgs.append( (inputFilePath, outputFilePath) )

	workerPool = multiprocessing.Pool(NUM_WORKERS)
	workerPool.map_async(processFile, filePathArgs, TASK_PARTITION_SIZE)	
	workerPool.close()
	workerPool.join()

	outputFileList = os.listdir(outputDir)

	final = open(outputDir+"FINAL.txt", "a")

	totalDict = collections.defaultdict(float)
	numberDict = collections.defaultdict(int)	


	for ofn in outputFileList:
		f = open(outputDir+"/"+ofn, 'r')
		for line in f:
			tokens = line.split('\t')
			key = tokens[0]
			value = tokens[1]
			number = tokens[2]

			totalDict[key] += float(value)
			numberDict[key] += int(number)


	#print totalDict
	for key in sorted(totalDict.keys()):
		final.write(str(key) + "\t" + str(totalDict[key]/numberDict[key]) + "\n")
		
			
def processFile(filePathArgs):
	
	try:
		inputFilePath = filePathArgs[0]
		outputFilePath = filePathArgs[1]

		print timestamp() + " Starting processing for " + inputFilePath + " to " + outputFilePath

		inputFileHandle = open(inputFilePath,"r")
		outputFileHandle = open(outputFilePath, 'a+')

		translate(inputFileHandle, outputFileHandle)
	
		inputFileHandle.close()
		outputFileHandle.close()

		print timestamp() + " Finished processing for " + inputFilePath + " in " + outputFilePath
	
	except:
		traceback.print_exc(limit=3, file=sys.stdout)
		sys.stdout.flush()


def translate(inputFileHandle, outputFileHandle):

	durationDict = collections.defaultdict(float)
	percentDict = collections.defaultdict(list)
	totalDict = collections.defaultdict(float)
	offsetList = []
	firstUser = True
	lineN = 0

	for line in inputFileHandle:
		lineN += 1		
		tokens = line.split('\t')

		if not firstUser:
			if tokens[0] <> username:
				for x in offsetList:
					#print offsetList				
					for ns in nsList:
						key = ns+str(x)
						if key not in durationDict.keys():
							percentDict[startyear+key].append(0) 
				for key in durationDict.keys():
					if "Total" not in key:
						percentDict[startyear+key].append(durationDict[key]/float(totalDict[key[-1]]))
				offsetList = []
				durationDict.clear()
				totalDict.clear()

		try:
			username = tokens[0]
			duration = tokens[1]
			namespace = tokens[2]
			startyear = tokens[7]
			offset = tokens[8]
		except:
			print inputFileHandle, " ", lineN, " Failed"
		if offset not in offsetList:
			offsetList.append(offset)
		firstUser = False

		durationDict[namespace+offset] += float(duration)
		totalDict[offset] += float(duration)


	for x in offsetList:				
		for ns in nsList:
			key = ns+str(x)
			if key not in durationDict.keys():
				percentDict[startyear+key].append(0) 

	for key in durationDict.keys():
		if "Total" not in key:
			percentDict[startyear+key].append(durationDict[key]/float(totalDict[key[-1]]))					

	

	for key in sorted(percentDict.keys()):
		total = float(0)
		#print key, percentDict[key]		
		for value in percentDict[key]:
			total += float(value)
		outputFileHandle.write(key + "\t" + str(total) + "\t" + str(len(percentDict[key])) + "\n") 
		

def timestamp():
	return datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')

def timestampToDate(dateString):
	return datetime.datetime.strptime(dateString.strip(), '%Y-%m-%d %H:%M:%S')

def computeYear(dateString):
	return str(timestampToDate(dateString).year)

def computeOffsetYear(startYear, dateString):
	return str(int(computeYear(dateString)) - int(startYear))

def parseLine(line):

	tokens = line.split('\t')
	
	fields = {}

	fields['username'] = tokens[0]
	fields['timestamp'] = tokens[1]
	fields['namespace'] = tokens[2]
	fields['start'] = tokens[3]
	fields['end'] = tokens[4]
	
	
	return fields


if __name__ == '__main__':
	main()


	
